'''
通过站长平台抓取 关键词指数和竞争对手
'''
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
import re,json,redis,time
from urllib import parse

pool = redis.ConnectionPool(host='地址', port=6379, password='密码', db='数据表') #redis 链接

'''
id  数据库中的id，方便更新
keyword  需要查询的关键词
table    数据表，不需要写前缀，db文件中已设置
redis_val  redis数据，错误异常后 重新执行
'''
def select_price(id,keyword,table,redis_val):
    global pool
    r = redis.Redis(connection_pool=pool)
    keyword = parse.quote(keyword)  # 关键词加码
    headers = {
        'Referer': "http://tool.chinaz.com/kwevaluate?kw=%s" % (str(keyword))
    }
    try:
        data = requests.get("http://tool.chinaz.com/kwevaluate?kw=%s"%(str(keyword)), timeout=10)
        getresult = re.findall('getresult\((.+)\)', data.text)
        print("获取get")
        getresult[0]=getresult[0].replace("'", "");
        key = getresult[0].split(',')
        url = "http://tool.chinaz.com/AjaxSeo.aspx?t=kwevaluate&index="+str(key[1])+"&kw="+str(keyword)+"&des="+str(key[3])+"&_=%d"%(int(time.time()))
        data = requests.get(url,headers=headers,timeout=5)
        first = re.findall('first:"(.+)",bit', data.text)
        data = requests.get("http://index.chinaz.com/?words=%s"%(str(keyword)), headers=headers, timeout=5)
        pc = re.findall('"pc":(.+),"pc_change"', data.text)
        mobile = re.findall('"mobile":(.+),"mobile_change"', data.text)
        Rank = int(pc[0])+int(mobile[0])
        print("竞争对手数量"+str(first[0]))
        print("关键词指数"+str(Rank))
    except:
        r.lpush('first_rank', redis_val)  # 失败数据写入redis
        return False




if __name__=="__main__":
    r = redis.Redis(connection_pool=pool)
    p = Pool(2)  
    while True:
        try:
            redis_val = r.rpop("first_rank")
            if redis_val:
                redis_val = redis_val.decode()
                task = redis_val.split('^')
                # 如果搜索任务没有搜索标识，直接返回0
                p.apply_async(select_price, args=(str(task[0]), task[1],task[2], redis_val))
        except:
            pass
    p.close()  # 关闭进程池,不在接收新的任务
    p.join()  # 等待子进程全部运行完成，执行后续操作
